package dataset.generation;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.util.Span;

import org.itc.irst.tcc.sre.util.PorterStemmer;
import org.mitre.jawb.io.SgmlDocument;
import org.mitre.jawb.io.SgmlElement;

import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;

public class JSREFileCreator {

	private static SentenceDetectorME sentenceDetector;
	
	private static final String NATURAL_DISASTER = "NaturalDisaster";

	private static Map<String, Hashtable<String,Set<String>>> argumentsCombination = new Hashtable<String, Hashtable<String,Set<String>>>();
	
	private static Hashtable<String, Set<String>> allArgument = new Hashtable<String, Set<String>>();

	private static Hashtable<String, String> replacements = new Hashtable<String, String>();
	
	public static String[] Datasets = {"AFFILIATE-PARTNER","ASSOCIATE","AT","BASED-IN","CITIZEN-OF","CLIENT","FOUNDER","GENERAL-STAFF","GRANDPARENT",
		"LOCATED", "MANAGEMENT","MEMBER","NEAR","OTHER","OTHER-PERSONAL","OTHER-PROFESSIONAL","OTHER-RELATIVE","OWNER","PARENT",
		"PART","PART-OF","RELATIVE-LOCATION","RESIDENCE","ROLE","SIBLING","SOC","SPOUSE","SUBSIDIARY","NaturalDisaster"};

	private static Hashtable<String,PrintStream> printStreams;
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
//		Run train for both.
//		
//		Run test here only for NaturalDisasters.
		
		sentenceDetector = new SentenceDetectorME(new SentenceModel(new FileInputStream(new File("en-sent.bin"))));
		
		 // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution 
	    Properties props = new Properties();
//	    props.put("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref");
	    
	    props.put("annotators", "tokenize, ssplit, pos, lemma");
	    	    
	    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
		
		String fileCombinationsName = "/home/pjbarrio/Dataset/SGML-ACE/COMBINATIONS";
		
		loadFileForCombinations(new File(fileCombinationsName));
		
		String fileReplacementsName = "/home/pjbarrio/Dataset/SGML-ACE/REPLACEMENTS";
		
		loadFileForReplacements(new File(fileReplacementsName));
		
		PorterStemmer porterStemmer = PorterStemmer.getStemmer();
		
		for (int dataset = 0; dataset < Datasets.length; dataset++) {
			
			resetPrintStream();
			
			System.gc();
			
			System.err.println(Datasets[dataset]);
			
			String[] RelationTags = getRelationTags(Datasets[dataset]);
			
			String extensionFile = getExtensionFile(Datasets[dataset]);
			
			int[] splits = getSplits(Datasets[dataset]);
			
			for (int spl = 0; spl < splits.length; spl++) {

				String filesFolder = "trainingDirectory";
				
				String fileFor = "train";
				
				boolean separateClasses = true;
				
//				String rootDirectory = getDirectory(Datasets[dataset]);
//				
//				String directoryName = rootDirectory + splits[spl]+"/taggedtestingDirectory/";
//				
//				String outputFile = rootDirectory + splits[spl]+"/" + Datasets[dataset];// + ".jsre.train";
//				
//				String type = ".jsre.test";
				
//				System.setOut(new PrintStream(new File(outputFile)));
				
//				System.setErr(new PrintStream(new File(outputFile + ".error")));
				
				File directory = new File(getFilesDirectory(Datasets[dataset],splits[spl],filesFolder));
				
				File[] files = directory.listFiles();
		
		//		Arrays.sort(files);
		
				int documentId = 0;
				
				for (File file : files) {
					
					documentId++;
					
					System.gc();
					
					if (!file.getName().endsWith(extensionFile))
						continue;
		
					System.err.println(file.getName());
					
					SgmlDocument sgmlDocument = new SgmlDocument(new FileReader(file));
					
					Hashtable<String, List<SgmlElement>> sgmlElements = getSgmlElements(sgmlDocument,allArgument.get(Datasets[dataset]).toArray(new String[0]));
					
					Hashtable<String, List<SgmlElement>> sgmlRelationElements = getSgmlElements(sgmlDocument, RelationTags);
					
					List<Segment> sentences = getRelevantSentences(sgmlDocument,argumentsCombination.get(Datasets[dataset]),sgmlElements,allArgument.get(Datasets[dataset]).toArray(new String[0])); 
		
					int sentenceId = 0;
					
					for (Segment segment : sentences) {
						
						sentenceId++;
						
						List<SgmledSegment> combinations = generateCombinations(segment,argumentsCombination.get(Datasets[dataset]),sgmlElements,allArgument.get(Datasets[dataset]).toArray(new String[0]));
						
						Hashtable<String, List<SgmlElement>> filteredRelations = filter(sgmlRelationElements, segment, RelationTags);
						
						int variationId = 0;
						
						for (SgmledSegment sgmledSegment : combinations) {
							
							
							variationId++;

							// create an empty Annotation just with the given text
						    Annotation sentence = new Annotation(sgmledSegment.getText());
						    
						    // run all Annotators on this text
						    pipeline.annotate(sentence);
							
							SgmlElement first_element = sgmledSegment.getRelevantSgmlElement(1);
							
							SgmlElement second_element = sgmledSegment.getRelevantSgmlElement(2);
							
//							List<SgmlElement> tobeHyphened = getToBeHyphenedSgmlElements(sgmledSegment.getSgmlElements(),first_element,second_element);
	
							if (contains(first_element, second_element) || contains(second_element,first_element)) //Problems to hyphenize.
								continue;
							
							List<CoreLabel> tokens = sentence.get(TokensAnnotation.class);
														
							Integer[] firstText = generateArray(first_element,tokens, sgmledSegment.getOffset());
							
							Integer[] secondText = generateArray(second_element,tokens, sgmledSegment.getOffset());
							
							if (overlapped(firstText,secondText) || overlapped(secondText,firstText)) //<GPE>United	Kingdom</GPE>-<LOCATION>Caribbean</LOCATION>
								continue;
							
							String firstPOSTag = getPOSTag(sgmledSegment,first_element,pipeline,firstText[0].intValue());
							
							String secondPOSTag = getPOSTag(sgmledSegment,second_element,pipeline,secondText[0].intValue());
							
					    	String[] words = new String[tokens.size()];
					    	String[] lemmas = new String[tokens.size()];
					    	String[] pos = new String[tokens.size()];
					    	String[] entityType = new String[tokens.size()];
					    	String[] entityLabel = new String[tokens.size()];
					    	
					    	//initialize to avoid nulls
					    	
					    	int firstIndex = firstText[0];
					    	int secondIndex = secondText[0];
					    	
					    	words[firstIndex] = "";
					    	words[secondIndex] = "";
					    	
					    	int classification = getClass(sgmledSegment, filteredRelations, RelationTags[0],RelationTags[1]);
												    	
					    	for (int i = 0; i < tokens.size(); i++) {
								
					    		if (i >= firstIndex && i<=firstText[firstText.length-1]){
					    			
					    			words[firstIndex] = words[firstIndex] + "_" + tokens.get(i).get(TextAnnotation.class);
//					    			lemmas[firstText[0]] = words[firstText[0]];
					    			pos[firstIndex] = firstPOSTag;
					    			entityType[firstIndex] = first_element.getGid();
					    			entityLabel[firstIndex] = "T";
					    			
					    			
					    		} else if (i >= secondIndex && i<=secondText[secondText.length-1]){

					    			words[secondIndex] = words[secondIndex] + "_" + tokens.get(i).get(TextAnnotation.class);
//					    			lemmas[secondText[0]] = words[secondText[0]];
					    			pos[secondIndex] = secondPOSTag;
					    			entityType[secondIndex] = second_element.getGid();
					    			
					    			if (second_element.getGid().equals(first_element.getGid())){
					    				entityLabel[secondIndex] = "T";
					    			
					    				if (secondIndex < firstIndex){
					    					if (classification == 1){
					    						classification = 2; 
					    					}
					    				}
					    					
					    			}else{
					    				entityLabel[secondIndex] = "A";
					    			}
					    			
					    		}else{
					    			
					    			words[i] = tokens.get(i).get(TextAnnotation.class);
					    			lemmas[i] = porterStemmer.stem(words[i]);
					    			pos[i] = tokens.get(i).get(PartOfSpeechAnnotation.class);
					    			entityType[i] = "O";
					    			entityLabel[i] = "O";
					    			
					    		}
					    		
							}
					    	
					    	words[firstIndex] = words[firstIndex].substring(1);
					    	lemmas[firstIndex] = words[firstIndex];
					    	
//					    	System.err.println(firstIndex + " - " + tokens.get(firstIndex).get(TextAnnotation.class));
//					    	System.err.println(firstText[firstText.length-1] + " - " + tokens.get(firstText[firstText.length-1]).get(TextAnnotation.class));
//					    	System.err.println(secondIndex + " - " + tokens.get(secondIndex).get(TextAnnotation.class));
//					    	System.err.println(secondText[secondText.length-1] + " - " + tokens.get(secondText[secondText.length-1]).get(TextAnnotation.class));
					    	
					    	words[secondIndex] = words[secondIndex].substring(1);

					    	lemmas[secondIndex] = words[secondIndex];
					    	
					    	PrintStream w = getPrintStream(first_element.getGid(),second_element.getGid(),Datasets[dataset],splits[spl],classification, fileFor, separateClasses);
					    	
					    	w.print("\n" + classification + "\t" + generateId(documentId,sentenceId,variationId) + "\t");
					    	
					    	int index = 0;
					    	
					    	for (int i = 0; i < words.length; i++){					    		
					    		
					    		if (words[i] == null || words[i].trim().equals(""))
					    			continue;
					    		else {
					    			
					    			w.print(index + "&&" + words[i] + "&&" + lemmas[i] + "&&" + pos[i] + "&&" + entityType[i] + "&&" + entityLabel[i] + " ");
					    			
					    		}
					    		
					    		index++;
					    	}
							
						}
						
						
					}
							
				}
				
				
				}
			
				closePrintStreams();
			
			}
		
	}

	public static String getFilesDirectory(String dataset, int split,
			String filesFolder) {
		
		return getDirectory(dataset) + split + "/" + filesFolder + "/";
		
	}

	private static void closePrintStreams() {
		
		for(Enumeration<PrintStream> e = printStreams.elements() ; e.hasMoreElements() ; ){
			PrintStream printStream = e.nextElement();
			printStream.close();
		}	
		
	}

	private static void resetPrintStream() {
		
		printStreams = new Hashtable<String, PrintStream>();
		
	}

	private static PrintStream getPrintStream(String gid, String gid2, String dataset, int split, int classification, String fileFor, boolean separateClasses) throws FileNotFoundException {
		
		String id = getFormattedFile(gid,gid2,dataset,split,classification>0,fileFor, separateClasses);
				
		PrintStream ret = printStreams.get(id);
		
		if (ret == null){
			
			ret = new PrintStream(id);
			
			printStreams.put(id,ret);
		
		}
		
		return ret;
	}

	public static String getFormattedFile(String gid, String gid2,
			String dataset, int split, boolean classification, String fileFor, boolean separateClasses) {
		
		String outputFilePrefix = getDirectory(dataset) + split+ "/" + dataset;
				
		String id = getSuffix(gid,gid2,split,classification,separateClasses);

		return outputFilePrefix + "_" + id + ".jsre." + fileFor;
		
	}

	public static String getSplittedFormatedFile(String generatedPrefix,
			int split) {
		return generatedPrefix + "." + split;
	}
	
	private static String getSuffix(String gid, String gid2, int split, boolean classification, boolean separateClasses) {
		
		if (separateClasses)
			return gid + "_" + gid2 + "_" + split + "_" + classification;
		
		return gid + "_" + gid2 + "_" + split;
	
	}
	
	private static boolean overlapped(Integer[] first, Integer[] second) {
		
		if (first[0] <= second[0]){
			if (first[first.length-1] >= second[0])
				return true;
		}
		return false;
	}

	private static String generateId(int documentId, int sentenceId,
			int variationId) {
		return new String(documentId + "-" + sentenceId + "-" + variationId);
	}
	
	private static void loadFileForReplacements(File file) throws IOException {
		
		BufferedReader br = new BufferedReader(new FileReader(file));
		
		String line = br.readLine();
		
		while (line!=null){
			
			System.out.println(line);
			
			String[] split = line.split(" - ");
			
			String tag = split[0];
			
			String replacement = split[1];

			replacements .put(tag,replacement);
		
			line = br.readLine();
			
		}
		
		br.close();

		
		
	}

	private static String getPOSTag(SgmledSegment sgmledSegment,
			SgmlElement element, StanfordCoreNLP pipeline, int index) {
		
		int begin = element.getStart() - sgmledSegment.getOffset();

		int end = element.getEnd() - sgmledSegment.getOffset();

		String text = sgmledSegment.getText();
		
		while (Character.isSpaceChar(text.toCharArray()[begin])){
			begin++;
		}
		
		String first = text.substring(0,begin);
		
		first = first + replacements.get(element.getGid());
		
		first = first + text.substring(end);
		
		
		Annotation sentence = new Annotation(text);
	    
		pipeline.annotate(sentence);
		
		List<CoreLabel> tokens = sentence.get(TokensAnnotation.class);
		
		return tokens.get(index).get(PartOfSpeechAnnotation.class);
		
	}

	private static boolean contains(SgmlElement element,
			SgmlElement contained) {
		
		return (contained.getStart() >= element.getStart() && contained.getEnd() <= element.getEnd());
		
	}

	private static Integer[] generateArray(SgmlElement element,
			List<CoreLabel> tokens, int offset) {
		
		ArrayList<Integer> ret = new ArrayList<Integer>();
		
		int elementBegin = element.getStart() - offset;
		
		int elementEnd = element.getEnd() - offset;
		
		int index = 0;
		
		for (CoreLabel token : tokens) {

			int begin = token.beginPosition();
			
			int end = token.endPosition();

//			System.err.println(elementBegin + " - " + elementEnd + " - " + begin + " - " + end + " - " + token.get(TextAnnotation.class));
			
			if (end <= elementBegin || begin >= elementEnd){
				
				if (begin >= elementEnd){
					break;
				}
				
				index++;
				continue;
				
			}
			
	        ret.add(new Integer(index));        
			
			index++;
		}

		
		
		return ret.toArray(new Integer[0]);
		
	}

	private static String getExtensionFile(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return ".txt";
		}
	
		return ".sgm";
		
	}

	public static String getDirectory(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return "/home/pjbarrio/Dataset/NaturalDisaster/RelatedData/reverbData/";
		}
	
		return "/home/pjbarrio/Dataset/SGML-ACE-COREF/" + relation + "/";
		
		
	}

	public static void loadFileForCombinations(File file) throws IOException {
		
		BufferedReader br = new BufferedReader(new FileReader(file));
		
		String line = br.readLine();
		
		while (line!=null){
			
			System.out.println(line);
			
			String[] split = line.split(" - ");
			
			String relation = split[0];
			
			String first = split[1];
			
			String second = split[2];
			
			Hashtable<String, Set<String>> args = argumentsCombination.get(relation);
			
			if (args == null){
				
				args = new Hashtable<String, Set<String>>();
				
			}
			
			Set<String> firstArg = args.get(first);
			
			if (firstArg == null){
				
				firstArg = new HashSet<String>();
				
			}
			
			firstArg.add(second);			
			
			args.put(first,firstArg);
			
			argumentsCombination.put(relation, args);
			
			Set<String> argsAll = allArgument.get(relation);
			
			if (argsAll == null){
				
				argsAll = new HashSet<String>();
			}
			
			argsAll.add(first);
			
			argsAll.add(second);
			
			allArgument.put(relation,argsAll);
			
			line = br.readLine();
		
		}
		
		br.close();
		
	}

	public static int[] getSplits(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			int[] ret = {0,1,2,3,4,5,6,7,8,9};
			return ret;
		}
	
		int[] ret = {0};
		
		return ret;
		
	}

	private static String[] getRelationTags(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			String[] ret = {"atLocation","locationOf"};
			return ret;
		}
	
		String[] ret = {relation + "First",relation + "Second"};
		
		return ret;
	}

	private static int getClass(SgmledSegment sgmledSegment,
			Hashtable<String, List<SgmlElement>> filteredRelations, String firstTag, String secondTag) {
		// returns 1 if the entities are related. 0 otherwise.
		
		SgmlElement relevantElement1 = sgmledSegment.getRelevantSgmlElement(1);
		
		SgmlElement relevantElement2 = sgmledSegment.getRelevantSgmlElement(2);
				
		List<SgmlElement> relevantRelationFirst = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relevantRelationSecond = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relations = filteredRelations.get(firstTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement1)){
				relevantRelationFirst.add(relationSgmlElement);
			}
			
		}

		
		relations = filteredRelations.get(secondTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement2)){
				relevantRelationSecond.add(relationSgmlElement);
			}
			
		}
		
		
		if (relevantRelationFirst.size() > 0 && relevantRelationSecond.size()>0){
			
			if (sameRelation(relevantRelationFirst,relevantRelationSecond)){
				return 1;
			}
			
		}
		
		return 0;
	}

	private static boolean sameRelation(List<SgmlElement> relevantRelationFirst, List<SgmlElement> relevantRelationSecond) {
		
		for (SgmlElement sgmlElement : relevantRelationFirst) {
			
			String attribute = sgmlElement.getAttribute("id");
			
			for (SgmlElement sgmlElement2 : relevantRelationSecond) {
				
				String attribute2 = sgmlElement.getAttribute("id");
				
				if (attribute.equals(attribute2)){
					return true;
				}
				
			}
			
		}
		
		return false;
		
	}

	private static boolean matches(SgmlElement relationSgmlElement,
			SgmlElement sgmlElement) {
		
		return (relationSgmlElement.getStart() >= sgmlElement.getStart() && relationSgmlElement.getEnd() <= sgmlElement.getEnd()) 
				|| (sgmlElement.getStart()>=relationSgmlElement.getStart() && sgmlElement.getEnd() <= relationSgmlElement.getEnd());
		
	}

	private static List<SgmledSegment> generateCombinations(Segment segment, Hashtable<String, Set<String>> combination, Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {
		
		List<SgmledSegment> ret = new ArrayList<SgmledSegment>();
		
		Hashtable<String, List<SgmlElement>> containedSgmlElements = filter(sgmlElements,segment,tags);
	
		List<SgmlElement> sorted = sort(containedSgmlElements);

		Set<String> keyset = combination.keySet();
		
		for (String firstTag : keyset) {
			
			List<SgmlElement> tag_0 = containedSgmlElements.get(firstTag);
			
			Set<String> seconds = combination.get(firstTag);
			
			for (String second : seconds) {
				
				List<SgmlElement> tag_1 = containedSgmlElements.get(second);
				
				for (SgmlElement sgmlElement_0 : tag_0) {
					
					for (SgmlElement sgmlElement_1 : tag_1) {
						
						SgmledSegment aux = new SgmledSegment(segment.getText(), segment.getOffset(), sorted);
						
						aux.addSgmlElement(sgmlElement_0,1);
						
						aux.addSgmlElement(sgmlElement_1,2);
						
						ret.add(aux);
						
					}
					
				}
				
			}
		
		}
				
		
		return ret;
		
	}

	private static List<SgmlElement> sort(
			Hashtable<String, List<SgmlElement>> containedSgmlElements) {
		
		List<SgmlElement> totalSgmlElements = new ArrayList<SgmlElement>();
		
		for(Enumeration<String> e = containedSgmlElements.keys();e.hasMoreElements();){
			totalSgmlElements.addAll(containedSgmlElements.get(e.nextElement()));
		}

		Collections.sort(totalSgmlElements, new Comparator<SgmlElement>() {

			@Override
			public int compare(SgmlElement arg0, SgmlElement arg1) {
				
				return Integer.valueOf(arg0.getStart()).compareTo(arg1.getStart());
				
			}
		});
		
		return totalSgmlElements;
	}

	private static Hashtable<String, List<SgmlElement>> filter(
			Hashtable<String, List<SgmlElement>> sgmlElements, Segment segment,
			String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			List<SgmlElement> elements = sgmlElements.get(tags[i]);
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (SgmlElement sgmlElement : elements) {
				
				if (contains(segment, sgmlElement)){
					aux.add(sgmlElement);
				}
				
			}
			
			ret.put(tags[i], aux);
			
		}
		
		return ret;
	}

	private static List<Segment> getRelevantSentences(SgmlDocument sgmlDocument, Hashtable<String, Set<String>> combination, Hashtable<String, List<SgmlElement>> sgmlElements, String[] Tags) throws FileNotFoundException, IOException {
		
		String plainText = sgmlDocument.getSignalText();
		
		List<Segment> segments = getSegments(plainText);
		
		List<Segment> sentences = generateSentences(segments);
		
		List<Segment> fixedSentences = fixSentences(sentences,sgmlElements,Tags);
		
		List<Segment> filteredSenteces = filterSentences(sgmlElements,fixedSentences,combination);
		
		return filteredSenteces;
	}

	private static List<Segment> fixSentences(List<Segment> segments,
			Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {

		List<Segment> ret = new ArrayList<Segment>();

		for (int j = 0; j < segments.size(); j++) {
			
			Segment seg = segments.get(j);
			
			boolean changed = false;
			
			for (int i = 0; i < tags.length && !changed; i++) {
				
				List<SgmlElement> se = sgmlElements.get(tags[i]);
				
				for (SgmlElement sgmlElement : se) {
					
					if (contains(seg, sgmlElement) && !containsItAll(seg, sgmlElement)){
						
						String link = "";
						
						int initSec = segments.get(j+1).getOffset();
						int endPrev = seg.getOffset() + seg.getText().length();
						
						int add = initSec - endPrev;
						
						for (int k = 0; k < add; k++) {
						
							link = link + " ";
						
						}
						
						segments.set(j+1, new Segment(seg.getText() + link + segments.get(j+1).getText(), seg.getOffset()));
						
						changed = true;
						
						break;
						
					}
					
				}
				
			}
			
			if (!changed)
				ret.add(seg);
			
		}
		
		return ret;
		
	}

	private static List<Segment> filterSentences(
			Hashtable<String, List<SgmlElement>> sgmlElements,
			List<Segment> sentences, Hashtable<String, Set<String>> combination) {
		// return sentences that have both tags.
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		Set<String> keyset = combination.keySet();

		for (Segment sentence : sentences) {
					
			boolean isIn = false;
			
			for (String first : keyset) {
				
				Set<String> seconds = combination.get(first);
				
				for (String second : seconds) {
										
					if (contains(sentence,sgmlElements.get(first)) && contains(sentence,sgmlElements.get(second))){
						isIn = true;
					}
					
					
				}
				
			}
		
			if (isIn){
				ret.add(sentence);
			}
			
		}
	
		return ret;
	}

	private static boolean contains(Segment sentence, List<SgmlElement> list) {
		
		for (SgmlElement sgmlElement : list) {
			
			if (contains(sentence,sgmlElement)){
				return true;
			}
						
		}
		
		return false;
	}

	private static boolean containsItAll(Segment sentence, SgmlElement sgmlElement) {

		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getEnd() <= end){
			
			return true;
			
		}
		
		return false;
		
	}
	
	private static boolean contains(Segment sentence, SgmlElement sgmlElement) {
		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getStart() <= end){
			
			return true;
			
		}
		
		return false;
		
	}

	private static Hashtable<String, List<SgmlElement>> getSgmlElements(
			SgmlDocument sgmlDocument, String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (Iterator iterator = sgmlDocument.iterator(); iterator.hasNext();) {
				
				SgmlElement sgmlElement = (SgmlElement) iterator.next();
				
				if (sgmlElement.getGid().equals(tags[i])){
					
					aux.add(sgmlElement);
					
				}
				
			}
			
			ret.put(tags[i],aux);
			
		}
		
		return ret;
		
	}

	private static List<Segment> generateSentences(List<Segment> contentParagraph) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment segment : contentParagraph){
			
			ret.addAll(getSentences(segment));
			
		}

		return ret;
		
	}

	private static Collection<? extends Segment> getSentences(Segment segment) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		String sentences[] = sentenceDetector.sentDetect(segment.getText());
			
		Span[] spans = sentenceDetector.sentPosDetect(segment.getText());
		
		for (int i = 0; i < sentences.length; i++) {
			
			ret.add(new Segment(sentences[i],segment.getOffset() + spans[i].getStart()));
			
		}

		return ret;
		
	}

	
	private static List<Segment> getSegments(String text) throws IOException {
		
		BufferedReader br = new BufferedReader(new StringReader(text));
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
	    String acum = "";
	    
	    String line = br.readLine();
	     
	    int offset = 0;
	    
	    int newOffset = 0;
	    
	    while (line != null){
	    	
	    	if (line.trim().equals("")){
	    		
	    		if (!acum.trim().equals("")){
	    				    			
	    			Segment s = new Segment(acum,offset);
	    			
	    			ret.add(s);
	    		
	    		}
	    		
	    		acum = "";
	    	
	    	} else {
	    		
	    		if (acum.trim().equals("")){  //First
	    			
	    			acum = line;
	    			
	    			offset = newOffset;
	    			
	    		} else {
		    		acum = acum + " " + line;
	    		}
	    		
	    	}
	    	
	    	newOffset += line.length() + 1;
	    	
	    	line = br.readLine();
	    
	    }
	    
	    if (!acum.trim().equals("")){
	    	
	    	Segment s = new Segment(acum,offset);
	    	
	    	ret.add(s);
	    	
	    }
	    
	    br.close();
	    
	    return ret;
		
	}

	public static String[] getFirstArgument(String dataset) {
		return argumentsCombination.get(dataset).keySet().toArray(new String[0]);
	}

	public static String[] getSecondArgument(String dataset, String first) {
		
		return argumentsCombination.get(dataset).get(first).toArray(new String[0]);
		
	}

	
	
}
